Load and clean data

cloud_data_orig <- load_cloud_data()
cloud_data_ls <- clean_cloud_data(cloud_data_orig)
cloud_data <- dplyr::bind_rows(cloud_data_ls, .id = "image")

Exploratory Data Analysis

Raw Images

raw_features <- c("DF", "CF", "BF", "AF", "AN")
for (var in c("label", raw_features)) {
  cat(sprintf("\n\n### %s\n\n", var))
  plt <- plot_cloud_data(cloud_data, var)
  vthemes::subchunkify(
    plt, i = subchunk_idx, fig_width = 10, fig_height = 4
  )
  subchunk_idx <- subchunk_idx + 1
}

label

DF

CF

BF

AF

AN

Correlations

for (image_id in unique(cloud_data$image)) {
  plt_df <- cloud_data |> 
    dplyr::filter(image == !!image_id)
  plt <- plot_pairs(
    plt_df, columns = c("DF", "CF", "BF", "AF", "AN"), 
    point_size = 0.1, subsample = 0.5, color = plt_df$label
  ) +
    ggplot2::scale_color_manual(values = cloud_colors) +
    ggplot2::scale_fill_manual(values = cloud_colors) +
    ggplot2::labs(title = sprintf("Image %s", image_id))
  vthemes::subchunkify(
    plt, i = subchunk_idx, fig_width = 10, fig_height = 10
  )
  subchunk_idx <- subchunk_idx + 1
}

Feature Engineering

engineered_features <- c("NDAI", "SD", "CORR")
for (var in c("label", engineered_features)) {
  cat(sprintf("\n\n### %s\n\n", var))
  plt <- plot_cloud_data(cloud_data, var)
  vthemes::subchunkify(
    plt, i = subchunk_idx, fig_width = 10, fig_height = 4
  )
  subchunk_idx <- subchunk_idx + 1
}

label

NDAI

SD

CORR

Prediction Modeling

Data Splitting

To mimic the process of how we obtain our future data (in this case, we expect to obtain completely new images), we leave out one full image for the test set. For cross-validation and the training-validation splits, we also perform clustered sampling, where we partition the images into contiguous blocks to maintain the integrity of the image. Below, we show the image blocks used in the data splitting scheme.

# divide image into contiguous chunks
cloud_data <- add_cloud_blocks(cloud_data_ls)
plt <- plot_cloud_data(cloud_data, var = "block_id")
plt

# save one image for testing
test_image_idx <- sample(unique(cloud_data$image), 1)
train_data_all <- cloud_data |>
  dplyr::filter(!(image %in% test_image_idx))
test_data_all <- cloud_data |>
  dplyr::filter(image %in% test_image_idx)
print(sprintf("Test image: %s", test_image_idx))
## [1] "Test image: 1"

Modeling v1

We next use the aforementioned data splitting scheme to both tune model hyperparameters and select the best model from the following candidates:

Models under consideration:

  • Logistic regression
  • Lasso regression (needs tuning)
  • Ridge regression (needs tuning)
  • Random forest (no tuning; using default hyperparameters)

Note that within glmnet::cv.glmnet, there is an interior cross-validation loop to tune the hyperparameters for LASSO and ridge, and by explicitly setting the foldid, we ensure that the cross-validation is done using our clustered sampling scheme by image block. If the R function doesn’t have a built-in CV option, we would need to code this up ourselves (or using other functions like from the caret R package). Within glmnet::cv.glmnet, there is also a re-fitting step, where the best hyperparameters are used to fit the model on the full training set.

keep_vars <- c("DF", "CF", "BF", "AF", "AN", "binary_label")
train_data <- cloud_data |>
  dplyr::filter(!(image %in% test_image_idx))
test_data <- cloud_data |>
  dplyr::filter(image %in% test_image_idx)

# evaluate validation error for various models
valid_auroc_ls <- list()
valid_auprc_ls <- list()
valid_preds_ls <- list()
for (fold in unique(train_data_all$block_id)) {
  # do data split
  cv_train_data_all <- train_data_all |> 
    dplyr::filter(block_id != !!fold)
  cv_train_data <- cv_train_data_all |> 
    dplyr::select(tidyselect::all_of(keep_vars))
  cv_valid_data_all <- train_data_all |> 
    dplyr::filter(block_id == !!fold)
  cv_valid_data <- cv_valid_data_all |> 
    dplyr::select(tidyselect::all_of(keep_vars))
  
  # fit logistic regression
  log_fit <- glm(
    binary_label ~ ., data = cv_train_data, family = "binomial"
  )
  log_preds <- predict(log_fit, cv_valid_data, type = "response")
  
  # fit and evaluate lasso regression
  lasso_fit <- glmnet::cv.glmnet(
    x = as.matrix(cv_train_data |> dplyr::select(-binary_label)),
    y = cv_train_data$binary_label,
    family = "binomial",
    alpha = 1,
    foldid = as.numeric(as.factor(cv_train_data_all$block_id))
  )
  lasso_preds <- predict(
    lasso_fit, 
    as.matrix(cv_valid_data |> dplyr::select(-binary_label)), 
    s = "lambda.min", # or "lambda.1se"
    type = "response"
  )
  
  # fit and evaluate ridge regression
  ridge_fit <- glmnet::cv.glmnet(
    x = as.matrix(cv_train_data |> dplyr::select(-binary_label)),
    y = cv_train_data$binary_label,
    family = "binomial",
    alpha = 0,
    foldid = as.numeric(as.factor(cv_train_data_all$block_id))
  )
  ridge_preds <- predict(
    ridge_fit, 
    as.matrix(cv_valid_data |> dplyr::select(-binary_label)), 
    s = "lambda.min", # or "lambda.1se"
    type = "response"
  )
  
  # fit random forest
  rf_fit <- ranger::ranger(
    binary_label ~ ., data = cv_train_data, 
    probability = TRUE, verbose = FALSE
  )
  rf_preds <- predict(rf_fit, cv_valid_data)$predictions[, 2]
  
  # evaluate predictions
  preds_ls <- list(
    "logistic" = log_preds,
    "lasso" = lasso_preds,
    "ridge" = ridge_preds,
    "rf" = rf_preds
  )
  valid_auroc_ls[[fold]] <- purrr::map(
    preds_ls,
    ~ yardstick::roc_auc_vec(
      truth = cv_valid_data$binary_label, 
      estimate = c(.x), 
      event_level = "second"
    )
  ) |>
    dplyr::bind_rows(.id = "method")
  valid_auprc_ls[[fold]] <- purrr::map(
    preds_ls,
    ~ yardstick::pr_auc_vec(
      truth = cv_valid_data$binary_label, 
      estimate = c(.x), 
      event_level = "second"
    )
  ) |>
    dplyr::bind_rows(.id = "method")
  
  # save fold predictions for future investigation
  valid_preds_ls[[fold]] <- cv_valid_data_all |> 
    dplyr::bind_cols(preds_ls)
}

# examine validation accuracy
valid_preds_ls1 <- valid_preds_ls
valid_auroc_df <- dplyr::bind_rows(valid_auroc_ls, .id = "fold")
mean_valid_auroc_df <- valid_auroc_df |> 
  dplyr::summarise(dplyr::across(-fold, ~ mean(.x, na.rm = TRUE)))
valid_auprc_df <- dplyr::bind_rows(valid_auprc_ls, .id = "fold")
mean_valid_auprc_df <- valid_auprc_df |> 
  dplyr::summarise(dplyr::across(-fold, ~ mean(.x, na.rm = TRUE)))

# evaluate best model on test set
train_data <- train_data_all |> 
  dplyr::select(tidyselect::all_of(keep_vars))
test_data <- test_data_all |>
  dplyr::select(tidyselect::all_of(keep_vars))
best_fit <- ranger::ranger(
  binary_label ~ ., data = train_data, probability = TRUE, verbose = FALSE
)
test_preds <- predict(best_fit, test_data)$predictions[, 2]
test_auroc <- yardstick::roc_auc_vec(
  truth = test_data$binary_label, 
  estimate = test_preds, 
  event_level = "second"
)
test_auprc <- yardstick::pr_auc_vec(
  truth = test_data$binary_label, 
  estimate = test_preds, 
  event_level = "second"
)

auroc_df <- mean_valid_auroc_df |>
  dplyr::rename_with(~ sprintf("Validation %s AUROC", stringr::str_to_title(.x))) |> 
  dplyr::mutate(
    `Test AUROC` = test_auroc
  )
auprc_df <- mean_valid_auprc_df |>
  dplyr::rename_with(~ sprintf("Validation %s AUPRC", stringr::str_to_title(.x))) |> 
  dplyr::mutate(
    `Test AUPRC` = test_auprc
  )

vthemes::pretty_kable(
  valid_auroc_df, caption = "Fold-wise Validation AUROC for Various Models"
)
Table 1: Fold-wise Validation AUROC for Various Models
fold logistic lasso ridge rf
6 0.885 0.885 0.886 0.881
5 0.791 0.798 0.811 0.713
7 0.817 0.814 0.815 0.851
8 0.907 0.908 0.912 0.928
10 0.378 0.378 0.382 0.493
9 0.681 0.681 0.678 0.578
11 0.130 0.129 0.124 0.602
12 0.652 0.653 0.655 0.843
vthemes::pretty_kable(
  auroc_df, caption = "Overall AUROC Prediction Performance"
)
Table 1: Overall AUROC Prediction Performance
Validation Logistic AUROC Validation Lasso AUROC Validation Ridge AUROC Validation Rf AUROC Test AUROC
0.655 0.656 0.658 0.736 0.758
vthemes::pretty_kable(
  valid_auprc_df, caption = "Fold-wise Validation AUPRC for Various Models"
)
Table 1: Fold-wise Validation AUPRC for Various Models
fold logistic lasso ridge rf
6 0.759 0.760 0.762 0.777
5 0.953 0.955 0.959 0.925
7 0.289 0.286 0.284 0.359
8 0.372 0.373 0.380 0.485
10 0.321 0.321 0.323 0.388
9 0.311 0.311 0.306 0.266
11 0.0117 0.0117 0.0116 0.0328
12 0.153 0.153 0.158 0.340
vthemes::pretty_kable(
  auprc_df, caption = "Overall AUPRC Prediction Performance"
)
Table 1: Overall AUPRC Prediction Performance
Validation Logistic AUPRC Validation Lasso AUPRC Validation Ridge AUPRC Validation Rf AUPRC Test AUPRC
0.396 0.396 0.398 0.446 0.327

How are we doing? Are we doing well? Any concerns?

Post-hoc investigations v1

Let’s look at the held-out folds where the methods didn’t perform so well and compare the predictions across methods.

plot_vars <- c(
  "binary_label", "logistic", "lasso", "ridge", "rf",
  "DF", "CF", "BF", "AF", "AN"#, "NDAI", "SD", "CORR"
)
for (fold in unique(train_data_all$block_id)) {
  cat(sprintf("\n\n#### Fold %s\n\n", fold))
  preds_df <- valid_preds_ls[[fold]]
  plt_ls <- list()
  for (var in plot_vars) {
    plt_ls[[var]] <- plot_cloud_data(preds_df, var)
  }
  plt <- patchwork::wrap_plots(plt_ls, ncol = 5)
  vthemes::subchunkify(
    plt, i = subchunk_idx, fig_width = 14, fig_height = 6
  )
  subchunk_idx <- subchunk_idx + 1
}

Fold 6

Fold 5

Fold 7

Fold 8

Fold 10

Fold 9

Fold 11

Fold 12

# fold <- "8" # good fold
# fold <- "10" # "11" # bad fold
# preds_df <- valid_preds_ls1[[fold]]
# plot_vars <- c(
#   # "label",
#   "binary_label", "logistic", "lasso", "ridge", "rf",
#   "DF", "CF", "BF", "AF", "AN"#, "NDAI", "SD", "CORR"
# )
# plt_ls <- list()
# for (var in plot_vars) {
#   plt_ls[[var]] <- plot_cloud_data(preds_df, var)
# }
# plt <- patchwork::wrap_plots(plt_ls, ncol = 5)
# plt

Modeling v2

Engineering features based upon prior or domain knowledge can greatly improve the accuracy of our models. In Shi et al. (2008), the authors engineered three additional features:

  • NDAI: measures difference in reflectance values between different radiance angles (or spectral bands)
  • CORR: measures correlation between the different radiance angles
  • SD: measures variability in reflectance values surrounding the pixel

Let’s repeat the previous analysis including these engineered features.

keep_vars <- c("NDAI", "SD", "CORR", keep_vars)
train_data <- cloud_data |>
  dplyr::filter(!(image %in% test_image_idx))
test_data <- cloud_data |>
  dplyr::filter(image %in% test_image_idx)

# evaluate validation error for various models
valid_auroc_ls <- list()
valid_auprc_ls <- list()
valid_preds_ls <- list()
for (fold in unique(train_data_all$block_id)) {
  # do data split
  cv_train_data_all <- train_data_all |> 
    dplyr::filter(block_id != !!fold)
  cv_train_data <- cv_train_data_all |> 
    dplyr::select(tidyselect::all_of(keep_vars))
  cv_valid_data_all <- train_data_all |> 
    dplyr::filter(block_id == !!fold)
  cv_valid_data <- cv_valid_data_all |> 
    dplyr::select(tidyselect::all_of(keep_vars))
  
  # fit logistic regression
  log_fit <- glm(
    binary_label ~ ., data = cv_train_data, family = "binomial"
  )
  log_preds <- predict(log_fit, cv_valid_data, type = "response")
  
  # fit and evaluate lasso regression
  lasso_fit <- glmnet::cv.glmnet(
    x = as.matrix(cv_train_data |> dplyr::select(-binary_label)),
    y = cv_train_data$binary_label,
    family = "binomial",
    alpha = 1,
    foldid = as.numeric(as.factor(cv_train_data_all$block_id))
  )
  lasso_preds <- predict(
    lasso_fit, 
    as.matrix(cv_valid_data |> dplyr::select(-binary_label)), 
    s = "lambda.min", # or "lambda.1se"
    type = "response"
  )
  
  # fit and evaluate ridge regression
  ridge_fit <- glmnet::cv.glmnet(
    x = as.matrix(cv_train_data |> dplyr::select(-binary_label)),
    y = cv_train_data$binary_label,
    family = "binomial",
    alpha = 0,
    foldid = as.numeric(as.factor(cv_train_data_all$block_id))
  )
  ridge_preds <- predict(
    ridge_fit, 
    as.matrix(cv_valid_data |> dplyr::select(-binary_label)), 
    s = "lambda.min", # or "lambda.1se"
    type = "response"
  )
  
  # fit random forest
  rf_fit <- ranger::ranger(
    binary_label ~ ., data = cv_train_data, 
    probability = TRUE, verbose = FALSE
  )
  rf_preds <- predict(rf_fit, cv_valid_data)$predictions[, 2]
  
  # evaluate predictions
  preds_ls <- list(
    "logistic" = log_preds,
    "lasso" = lasso_preds,
    "ridge" = ridge_preds,
    "rf" = rf_preds
  )
  valid_auroc_ls[[fold]] <- purrr::map(
    preds_ls,
    ~ yardstick::roc_auc_vec(
      truth = cv_valid_data$binary_label, 
      estimate = c(.x), 
      event_level = "second"
    )
  ) |>
    dplyr::bind_rows(.id = "method")
  valid_auprc_ls[[fold]] <- purrr::map(
    preds_ls,
    ~ yardstick::pr_auc_vec(
      truth = cv_valid_data$binary_label, 
      estimate = c(.x), 
      event_level = "second"
    )
  ) |>
    dplyr::bind_rows(.id = "method")
  
  # save fold predictions for future investigation
  valid_preds_ls[[fold]] <- cv_valid_data_all |> 
    dplyr::bind_cols(preds_ls)
}

# examine validation accuracy
valid_preds_ls2 <- valid_preds_ls
new_valid_auroc_df <- dplyr::bind_rows(valid_auroc_ls, .id = "fold") |> 
  dplyr::rename_with(~ sprintf("%s (new)", stringr::str_to_title(.x))) |> 
  dplyr::rename(fold = "Fold (new)") |> 
  dplyr::left_join(valid_auroc_df, by = "fold")
new_mean_valid_auroc_df <- new_valid_auroc_df |> 
  dplyr::summarise(dplyr::across(-fold, ~ mean(.x, na.rm = TRUE)))
new_valid_auprc_df <- dplyr::bind_rows(valid_auprc_ls, .id = "fold") |> 
  dplyr::rename_with(~ sprintf("%s (new)", stringr::str_to_title(.x))) |> 
  dplyr::rename(fold = "Fold (new)") |> 
  dplyr::left_join(valid_auprc_df, by = "fold")
new_mean_valid_auprc_df <- new_valid_auprc_df |> 
  dplyr::summarise(dplyr::across(-fold, ~ mean(.x, na.rm = TRUE)))

# evaluate best model on test set
train_data <- train_data_all |> 
  dplyr::select(tidyselect::all_of(keep_vars))
test_data <- test_data_all |>
  dplyr::select(tidyselect::all_of(keep_vars))
best_fit <- ranger::ranger(
  binary_label ~ ., data = train_data, probability = TRUE, verbose = FALSE
)
test_preds <- predict(best_fit, test_data)$predictions[, 2]
test_auroc <- yardstick::roc_auc_vec(
  truth = test_data$binary_label, 
  estimate = test_preds, 
  event_level = "second"
)
test_auprc <- yardstick::pr_auc_vec(
  truth = test_data$binary_label, 
  estimate = test_preds, 
  event_level = "second"
)

new_auroc_df <- new_mean_valid_auroc_df |>
  dplyr::rename_with(~ sprintf("Validation %s AUROC", stringr::str_to_title(.x))) |> 
  dplyr::mutate(
    `Test AUROC` = test_auroc
  )
new_auprc_df <- new_mean_valid_auprc_df |>
  dplyr::rename_with(~ sprintf("Validation %s AUPRC", stringr::str_to_title(.x))) |> 
  dplyr::mutate(
    `Test AUPRC` = test_auprc
  )

vthemes::pretty_kable(
  new_valid_auroc_df, caption = "Fold-wise Validation AUROC for Various Models"
)
Table 2: Fold-wise Validation AUROC for Various Models
fold Logistic (new) Lasso (new) Ridge (new) Rf (new) logistic lasso ridge rf
6 0.875 0.875 0.880 0.912 0.885 0.885 0.886 0.881
5 0.820 0.821 0.827 0.791 0.791 0.798 0.811 0.713
7 0.876 0.874 0.862 0.919 0.817 0.814 0.815 0.851
8 0.945 0.945 0.955 0.952 0.907 0.908 0.912 0.928
10 0.430 0.427 0.383 0.635 0.378 0.378 0.382 0.493
9 0.732 0.733 0.726 0.672 0.681 0.681 0.678 0.578
11 0.342 0.327 0.244 0.834 0.130 0.129 0.124 0.602
12 0.788 0.786 0.763 0.901 0.652 0.653 0.655 0.843
vthemes::pretty_kable(
  new_auroc_df, caption = "Overall AUROC Prediction Performance"
)
Table 2: Overall AUROC Prediction Performance
Validation Logistic (New) AUROC Validation Lasso (New) AUROC Validation Ridge (New) AUROC Validation Rf (New) AUROC Validation Logistic AUROC Validation Lasso AUROC Validation Ridge AUROC Validation Rf AUROC Test AUROC
0.726 0.724 0.705 0.827 0.655 0.656 0.658 0.736 0.821
vthemes::pretty_kable(
  new_valid_auprc_df, caption = "Fold-wise Validation AUPRC for Various Models"
)
Table 2: Fold-wise Validation AUPRC for Various Models
fold Logistic (new) Lasso (new) Ridge (new) Rf (new) logistic lasso ridge rf
6 0.759 0.759 0.767 0.821 0.759 0.760 0.762 0.777
5 0.963 0.964 0.965 0.948 0.953 0.955 0.959 0.925
7 0.365 0.362 0.336 0.507 0.289 0.286 0.284 0.359
8 0.502 0.504 0.512 0.527 0.372 0.373 0.380 0.485
10 0.344 0.343 0.324 0.572 0.321 0.321 0.323 0.388
9 0.379 0.379 0.358 0.355 0.311 0.311 0.306 0.266
11 0.0168 0.0159 0.0132 0.0737 0.0117 0.0117 0.0116 0.0328
12 0.242 0.238 0.203 0.601 0.153 0.153 0.158 0.340
vthemes::pretty_kable(
  new_auprc_df, caption = "Overall AUPRC Prediction Performance"
)
Table 2: Overall AUPRC Prediction Performance
Validation Logistic (New) AUPRC Validation Lasso (New) AUPRC Validation Ridge (New) AUPRC Validation Rf (New) AUPRC Validation Logistic AUPRC Validation Lasso AUPRC Validation Ridge AUPRC Validation Rf AUPRC Test AUPRC
0.446 0.445 0.435 0.551 0.396 0.396 0.398 0.446 0.400

Post-hoc investigations v2

Let’s look at the held-out folds where the methods didn’t perform so well and compare the predictions across methods.

plot_vars <- c(
  "binary_label", "logistic", "lasso", "ridge", "rf",
  "DF", "CF", "BF", "AF", "AN", "NDAI", "SD", "CORR"
)
for (fold in unique(train_data_all$block_id)) {
  cat(sprintf("\n\n#### Fold %s\n\n", fold))
  preds_df <- valid_preds_ls[[fold]]
  plt_ls <- list()
  for (var in plot_vars) {
    plt_ls[[var]] <- plot_cloud_data(preds_df, var)
  }
  plt <- patchwork::wrap_plots(plt_ls, ncol = 5)
  vthemes::subchunkify(
    plt, i = subchunk_idx, fig_width = 14, fig_height = 6
  )
  subchunk_idx <- subchunk_idx + 1
}

Fold 6

Fold 5

Fold 7

Fold 8

Fold 10

Fold 9

Fold 11

Fold 12

# fold <- "8" # good fold
# fold <- "10" # "11" # bad fold
# preds_df <- valid_preds_ls2[[fold]]
# plot_vars <- c(
#   # "label",
#   "binary_label", "logistic", "lasso", "ridge", "rf",
#   "DF", "CF", "BF", "AF", "AN", "NDAI", "SD", "CORR"
# )
# plt_ls <- list()
# for (var in plot_vars) {
#   plt_ls[[var]] <- plot_cloud_data(preds_df, var)
# }
# plt <- patchwork::wrap_plots(plt_ls, ncol = 5)
# plt

How can we do this post-hoc exploration if there are more than 8 variables?

Interpretations

To be continued…

LS0tCnRpdGxlOiAiU3VwZXJ2aXNlZCBMZWFybmluZyB3aXRoIENsb3VkcyBEYXRhIgphdXRob3I6ICJUaWZmYW55IFRhbmciCmRhdGU6ICJgciBTeXMuRGF0ZSgpYCIKb3V0cHV0OiAKICB2dGhlbWVzOjp2bW9kZXJuOgogICAgY29kZV9mb2xkaW5nOiBzaG93Ci0tLQoKYGBge3Igc2V0dXAsIGluY2x1ZGU9RkFMU0V9CmtuaXRyOjpvcHRzX2NodW5rJHNldCgKICBlY2hvID0gVFJVRSwKICB3YXJuaW5nID0gRkFMU0UsCiAgbWVzc2FnZSA9IEZBTFNFCikKCnNvdXJjZShoZXJlOjpoZXJlKCJSIiwgImxvYWQuUiIpKQpzb3VyY2UoaGVyZTo6aGVyZSgiUiIsICJjbGVhbi5SIikpCnNvdXJjZShoZXJlOjpoZXJlKCJSIiwgInBsb3QuUiIpKQoKc3ViY2h1bmtfaWR4IDwtIDEKc2V0LnNlZWQoMjQyKQoKY2xvdWRfY29sb3JzIDwtIGMoImRhcmsgZ3JlZW4iLCAibGlnaHQgYmx1ZSIsICJibGFjayIpCmBgYAoKIyBMb2FkIGFuZCBjbGVhbiBkYXRhIHsudGFic2V0IC50YWJzZXQtdm1vZGVybn0KCmBgYHtyfQpjbG91ZF9kYXRhX29yaWcgPC0gbG9hZF9jbG91ZF9kYXRhKCkKY2xvdWRfZGF0YV9scyA8LSBjbGVhbl9jbG91ZF9kYXRhKGNsb3VkX2RhdGFfb3JpZykKY2xvdWRfZGF0YSA8LSBkcGx5cjo6YmluZF9yb3dzKGNsb3VkX2RhdGFfbHMsIC5pZCA9ICJpbWFnZSIpCmBgYAoKIyBFeHBsb3JhdG9yeSBEYXRhIEFuYWx5c2lzIHsudGFic2V0IC50YWJzZXQtdm1vZGVybn0KCiMjIFJhdyBJbWFnZXMgey50YWJzZXQgLnRhYnNldC1waWxscyAudGFic2V0LXNxdWFyZX0KCmBgYHtyIHJlc3VsdHMgPSAiYXNpcyJ9CnJhd19mZWF0dXJlcyA8LSBjKCJERiIsICJDRiIsICJCRiIsICJBRiIsICJBTiIpCmZvciAodmFyIGluIGMoImxhYmVsIiwgcmF3X2ZlYXR1cmVzKSkgewogIGNhdChzcHJpbnRmKCJcblxuIyMjICVzXG5cbiIsIHZhcikpCiAgcGx0IDwtIHBsb3RfY2xvdWRfZGF0YShjbG91ZF9kYXRhLCB2YXIpCiAgdnRoZW1lczo6c3ViY2h1bmtpZnkoCiAgICBwbHQsIGkgPSBzdWJjaHVua19pZHgsIGZpZ193aWR0aCA9IDEwLCBmaWdfaGVpZ2h0ID0gNAogICkKICBzdWJjaHVua19pZHggPC0gc3ViY2h1bmtfaWR4ICsgMQp9CmBgYAoKIyMgQ29ycmVsYXRpb25zIHsudGFic2V0IC50YWJzZXQtcGlsbHMgLnRhYnNldC1zcXVhcmV9CgpgYGB7ciByZXN1bHRzID0gImFzaXMifQpmb3IgKGltYWdlX2lkIGluIHVuaXF1ZShjbG91ZF9kYXRhJGltYWdlKSkgewogIHBsdF9kZiA8LSBjbG91ZF9kYXRhIHw+IAogICAgZHBseXI6OmZpbHRlcihpbWFnZSA9PSAhIWltYWdlX2lkKQogIHBsdCA8LSBwbG90X3BhaXJzKAogICAgcGx0X2RmLCBjb2x1bW5zID0gYygiREYiLCAiQ0YiLCAiQkYiLCAiQUYiLCAiQU4iKSwgCiAgICBwb2ludF9zaXplID0gMC4xLCBzdWJzYW1wbGUgPSAwLjUsIGNvbG9yID0gcGx0X2RmJGxhYmVsCiAgKSArCiAgICBnZ3Bsb3QyOjpzY2FsZV9jb2xvcl9tYW51YWwodmFsdWVzID0gY2xvdWRfY29sb3JzKSArCiAgICBnZ3Bsb3QyOjpzY2FsZV9maWxsX21hbnVhbCh2YWx1ZXMgPSBjbG91ZF9jb2xvcnMpICsKICAgIGdncGxvdDI6OmxhYnModGl0bGUgPSBzcHJpbnRmKCJJbWFnZSAlcyIsIGltYWdlX2lkKSkKICB2dGhlbWVzOjpzdWJjaHVua2lmeSgKICAgIHBsdCwgaSA9IHN1YmNodW5rX2lkeCwgZmlnX3dpZHRoID0gMTAsIGZpZ19oZWlnaHQgPSAxMAogICkKICBzdWJjaHVua19pZHggPC0gc3ViY2h1bmtfaWR4ICsgMQp9CmBgYAoKIyMgRmVhdHVyZSBFbmdpbmVlcmluZyB7LnRhYnNldCAudGFic2V0LXBpbGxzIC50YWJzZXQtc3F1YXJlfQoKYGBge3IgcmVzdWx0cyA9ICJhc2lzIn0KZW5naW5lZXJlZF9mZWF0dXJlcyA8LSBjKCJOREFJIiwgIlNEIiwgIkNPUlIiKQpmb3IgKHZhciBpbiBjKCJsYWJlbCIsIGVuZ2luZWVyZWRfZmVhdHVyZXMpKSB7CiAgY2F0KHNwcmludGYoIlxuXG4jIyMgJXNcblxuIiwgdmFyKSkKICBwbHQgPC0gcGxvdF9jbG91ZF9kYXRhKGNsb3VkX2RhdGEsIHZhcikKICB2dGhlbWVzOjpzdWJjaHVua2lmeSgKICAgIHBsdCwgaSA9IHN1YmNodW5rX2lkeCwgZmlnX3dpZHRoID0gMTAsIGZpZ19oZWlnaHQgPSA0CiAgKQogIHN1YmNodW5rX2lkeCA8LSBzdWJjaHVua19pZHggKyAxCn0KYGBgCgojIFByZWRpY3Rpb24gTW9kZWxpbmcgey50YWJzZXQgLnRhYnNldC12bW9kZXJufQoKIyMgRGF0YSBTcGxpdHRpbmcgCgo8ZGl2IGNsYXNzPSJwYW5lbCBwYW5lbC1kZWZhdWx0IHBhZGRlZC1wYW5lbCI+ClRvIG1pbWljIHRoZSBwcm9jZXNzIG9mIGhvdyB3ZSBvYnRhaW4gb3VyIGZ1dHVyZSBkYXRhIChpbiB0aGlzIGNhc2UsIHdlIGV4cGVjdCB0byBvYnRhaW4gY29tcGxldGVseSBuZXcgaW1hZ2VzKSwgd2UgbGVhdmUgb3V0IG9uZSBmdWxsIGltYWdlIGZvciB0aGUgdGVzdCBzZXQuIEZvciBjcm9zcy12YWxpZGF0aW9uIGFuZCB0aGUgdHJhaW5pbmctdmFsaWRhdGlvbiBzcGxpdHMsIHdlIGFsc28gcGVyZm9ybSBjbHVzdGVyZWQgc2FtcGxpbmcsIHdoZXJlIHdlIHBhcnRpdGlvbiB0aGUgaW1hZ2VzIGludG8gY29udGlndW91cyBibG9ja3MgdG8gbWFpbnRhaW4gdGhlIGludGVncml0eSBvZiB0aGUgaW1hZ2UuIEJlbG93LCB3ZSBzaG93IHRoZSBpbWFnZSBibG9ja3MgdXNlZCBpbiB0aGUgZGF0YSBzcGxpdHRpbmcgc2NoZW1lLgo8L2Rpdj4KCmBgYHtyfQojIGRpdmlkZSBpbWFnZSBpbnRvIGNvbnRpZ3VvdXMgY2h1bmtzCmNsb3VkX2RhdGEgPC0gYWRkX2Nsb3VkX2Jsb2NrcyhjbG91ZF9kYXRhX2xzKQpwbHQgPC0gcGxvdF9jbG91ZF9kYXRhKGNsb3VkX2RhdGEsIHZhciA9ICJibG9ja19pZCIpCnBsdApgYGAKCmBgYHtyfQojIHNhdmUgb25lIGltYWdlIGZvciB0ZXN0aW5nCnRlc3RfaW1hZ2VfaWR4IDwtIHNhbXBsZSh1bmlxdWUoY2xvdWRfZGF0YSRpbWFnZSksIDEpCnRyYWluX2RhdGFfYWxsIDwtIGNsb3VkX2RhdGEgfD4KICBkcGx5cjo6ZmlsdGVyKCEoaW1hZ2UgJWluJSB0ZXN0X2ltYWdlX2lkeCkpCnRlc3RfZGF0YV9hbGwgPC0gY2xvdWRfZGF0YSB8PgogIGRwbHlyOjpmaWx0ZXIoaW1hZ2UgJWluJSB0ZXN0X2ltYWdlX2lkeCkKcHJpbnQoc3ByaW50ZigiVGVzdCBpbWFnZTogJXMiLCB0ZXN0X2ltYWdlX2lkeCkpCmBgYAoKIyMgTW9kZWxpbmcgdjEKCjxkaXYgY2xhc3M9InBhbmVsIHBhbmVsLWRlZmF1bHQgcGFkZGVkLXBhbmVsIj4KV2UgbmV4dCB1c2UgdGhlIGFmb3JlbWVudGlvbmVkIGRhdGEgc3BsaXR0aW5nIHNjaGVtZSB0byBib3RoIHR1bmUgbW9kZWwgaHlwZXJwYXJhbWV0ZXJzIGFuZCBzZWxlY3QgdGhlIGJlc3QgbW9kZWwgZnJvbSB0aGUgZm9sbG93aW5nIGNhbmRpZGF0ZXM6CgoqKk1vZGVscyB1bmRlciBjb25zaWRlcmF0aW9uOioqCgotIExvZ2lzdGljIHJlZ3Jlc3Npb24KLSBMYXNzbyByZWdyZXNzaW9uIChuZWVkcyB0dW5pbmcpCi0gUmlkZ2UgcmVncmVzc2lvbiAobmVlZHMgdHVuaW5nKQotIFJhbmRvbSBmb3Jlc3QgKG5vIHR1bmluZzsgdXNpbmcgZGVmYXVsdCBoeXBlcnBhcmFtZXRlcnMpCgpOb3RlIHRoYXQgd2l0aGluIGBnbG1uZXQ6OmN2LmdsbW5ldGAsIHRoZXJlIGlzIGFuIGludGVyaW9yIGNyb3NzLXZhbGlkYXRpb24gbG9vcCB0byB0dW5lIHRoZSBoeXBlcnBhcmFtZXRlcnMgZm9yIExBU1NPIGFuZCByaWRnZSwgYW5kIGJ5IGV4cGxpY2l0bHkgc2V0dGluZyB0aGUgYGZvbGRpZGAsIHdlIGVuc3VyZSB0aGF0IHRoZSBjcm9zcy12YWxpZGF0aW9uIGlzIGRvbmUgdXNpbmcgb3VyIGNsdXN0ZXJlZCBzYW1wbGluZyBzY2hlbWUgYnkgaW1hZ2UgYmxvY2suIElmIHRoZSBSIGZ1bmN0aW9uIGRvZXNuJ3QgaGF2ZSBhIGJ1aWx0LWluIENWIG9wdGlvbiwgd2Ugd291bGQgbmVlZCB0byBjb2RlIHRoaXMgdXAgb3Vyc2VsdmVzIChvciB1c2luZyBvdGhlciBmdW5jdGlvbnMgbGlrZSBmcm9tIHRoZSBgY2FyZXRgIFIgcGFja2FnZSkuIFdpdGhpbiBgZ2xtbmV0Ojpjdi5nbG1uZXRgLCB0aGVyZSBpcyBhbHNvIGEgcmUtZml0dGluZyBzdGVwLCB3aGVyZSB0aGUgYmVzdCBoeXBlcnBhcmFtZXRlcnMgYXJlIHVzZWQgdG8gZml0IHRoZSBtb2RlbCBvbiB0aGUgZnVsbCB0cmFpbmluZyBzZXQuCjwvZGl2PgoKYGBge3J9CmtlZXBfdmFycyA8LSBjKCJERiIsICJDRiIsICJCRiIsICJBRiIsICJBTiIsICJiaW5hcnlfbGFiZWwiKQp0cmFpbl9kYXRhIDwtIGNsb3VkX2RhdGEgfD4KICBkcGx5cjo6ZmlsdGVyKCEoaW1hZ2UgJWluJSB0ZXN0X2ltYWdlX2lkeCkpCnRlc3RfZGF0YSA8LSBjbG91ZF9kYXRhIHw+CiAgZHBseXI6OmZpbHRlcihpbWFnZSAlaW4lIHRlc3RfaW1hZ2VfaWR4KQoKIyBldmFsdWF0ZSB2YWxpZGF0aW9uIGVycm9yIGZvciB2YXJpb3VzIG1vZGVscwp2YWxpZF9hdXJvY19scyA8LSBsaXN0KCkKdmFsaWRfYXVwcmNfbHMgPC0gbGlzdCgpCnZhbGlkX3ByZWRzX2xzIDwtIGxpc3QoKQpmb3IgKGZvbGQgaW4gdW5pcXVlKHRyYWluX2RhdGFfYWxsJGJsb2NrX2lkKSkgewogICMgZG8gZGF0YSBzcGxpdAogIGN2X3RyYWluX2RhdGFfYWxsIDwtIHRyYWluX2RhdGFfYWxsIHw+IAogICAgZHBseXI6OmZpbHRlcihibG9ja19pZCAhPSAhIWZvbGQpCiAgY3ZfdHJhaW5fZGF0YSA8LSBjdl90cmFpbl9kYXRhX2FsbCB8PiAKICAgIGRwbHlyOjpzZWxlY3QodGlkeXNlbGVjdDo6YWxsX29mKGtlZXBfdmFycykpCiAgY3ZfdmFsaWRfZGF0YV9hbGwgPC0gdHJhaW5fZGF0YV9hbGwgfD4gCiAgICBkcGx5cjo6ZmlsdGVyKGJsb2NrX2lkID09ICEhZm9sZCkKICBjdl92YWxpZF9kYXRhIDwtIGN2X3ZhbGlkX2RhdGFfYWxsIHw+IAogICAgZHBseXI6OnNlbGVjdCh0aWR5c2VsZWN0OjphbGxfb2Yoa2VlcF92YXJzKSkKICAKICAjIGZpdCBsb2dpc3RpYyByZWdyZXNzaW9uCiAgbG9nX2ZpdCA8LSBnbG0oCiAgICBiaW5hcnlfbGFiZWwgfiAuLCBkYXRhID0gY3ZfdHJhaW5fZGF0YSwgZmFtaWx5ID0gImJpbm9taWFsIgogICkKICBsb2dfcHJlZHMgPC0gcHJlZGljdChsb2dfZml0LCBjdl92YWxpZF9kYXRhLCB0eXBlID0gInJlc3BvbnNlIikKICAKICAjIGZpdCBhbmQgZXZhbHVhdGUgbGFzc28gcmVncmVzc2lvbgogIGxhc3NvX2ZpdCA8LSBnbG1uZXQ6OmN2LmdsbW5ldCgKICAgIHggPSBhcy5tYXRyaXgoY3ZfdHJhaW5fZGF0YSB8PiBkcGx5cjo6c2VsZWN0KC1iaW5hcnlfbGFiZWwpKSwKICAgIHkgPSBjdl90cmFpbl9kYXRhJGJpbmFyeV9sYWJlbCwKICAgIGZhbWlseSA9ICJiaW5vbWlhbCIsCiAgICBhbHBoYSA9IDEsCiAgICBmb2xkaWQgPSBhcy5udW1lcmljKGFzLmZhY3Rvcihjdl90cmFpbl9kYXRhX2FsbCRibG9ja19pZCkpCiAgKQogIGxhc3NvX3ByZWRzIDwtIHByZWRpY3QoCiAgICBsYXNzb19maXQsIAogICAgYXMubWF0cml4KGN2X3ZhbGlkX2RhdGEgfD4gZHBseXI6OnNlbGVjdCgtYmluYXJ5X2xhYmVsKSksIAogICAgcyA9ICJsYW1iZGEubWluIiwgIyBvciAibGFtYmRhLjFzZSIKICAgIHR5cGUgPSAicmVzcG9uc2UiCiAgKQogIAogICMgZml0IGFuZCBldmFsdWF0ZSByaWRnZSByZWdyZXNzaW9uCiAgcmlkZ2VfZml0IDwtIGdsbW5ldDo6Y3YuZ2xtbmV0KAogICAgeCA9IGFzLm1hdHJpeChjdl90cmFpbl9kYXRhIHw+IGRwbHlyOjpzZWxlY3QoLWJpbmFyeV9sYWJlbCkpLAogICAgeSA9IGN2X3RyYWluX2RhdGEkYmluYXJ5X2xhYmVsLAogICAgZmFtaWx5ID0gImJpbm9taWFsIiwKICAgIGFscGhhID0gMCwKICAgIGZvbGRpZCA9IGFzLm51bWVyaWMoYXMuZmFjdG9yKGN2X3RyYWluX2RhdGFfYWxsJGJsb2NrX2lkKSkKICApCiAgcmlkZ2VfcHJlZHMgPC0gcHJlZGljdCgKICAgIHJpZGdlX2ZpdCwgCiAgICBhcy5tYXRyaXgoY3ZfdmFsaWRfZGF0YSB8PiBkcGx5cjo6c2VsZWN0KC1iaW5hcnlfbGFiZWwpKSwgCiAgICBzID0gImxhbWJkYS5taW4iLCAjIG9yICJsYW1iZGEuMXNlIgogICAgdHlwZSA9ICJyZXNwb25zZSIKICApCiAgCiAgIyBmaXQgcmFuZG9tIGZvcmVzdAogIHJmX2ZpdCA8LSByYW5nZXI6OnJhbmdlcigKICAgIGJpbmFyeV9sYWJlbCB+IC4sIGRhdGEgPSBjdl90cmFpbl9kYXRhLCAKICAgIHByb2JhYmlsaXR5ID0gVFJVRSwgdmVyYm9zZSA9IEZBTFNFCiAgKQogIHJmX3ByZWRzIDwtIHByZWRpY3QocmZfZml0LCBjdl92YWxpZF9kYXRhKSRwcmVkaWN0aW9uc1ssIDJdCiAgCiAgIyBldmFsdWF0ZSBwcmVkaWN0aW9ucwogIHByZWRzX2xzIDwtIGxpc3QoCiAgICAibG9naXN0aWMiID0gbG9nX3ByZWRzLAogICAgImxhc3NvIiA9IGxhc3NvX3ByZWRzLAogICAgInJpZGdlIiA9IHJpZGdlX3ByZWRzLAogICAgInJmIiA9IHJmX3ByZWRzCiAgKQogIHZhbGlkX2F1cm9jX2xzW1tmb2xkXV0gPC0gcHVycnI6Om1hcCgKICAgIHByZWRzX2xzLAogICAgfiB5YXJkc3RpY2s6OnJvY19hdWNfdmVjKAogICAgICB0cnV0aCA9IGN2X3ZhbGlkX2RhdGEkYmluYXJ5X2xhYmVsLCAKICAgICAgZXN0aW1hdGUgPSBjKC54KSwgCiAgICAgIGV2ZW50X2xldmVsID0gInNlY29uZCIKICAgICkKICApIHw+CiAgICBkcGx5cjo6YmluZF9yb3dzKC5pZCA9ICJtZXRob2QiKQogIHZhbGlkX2F1cHJjX2xzW1tmb2xkXV0gPC0gcHVycnI6Om1hcCgKICAgIHByZWRzX2xzLAogICAgfiB5YXJkc3RpY2s6OnByX2F1Y192ZWMoCiAgICAgIHRydXRoID0gY3ZfdmFsaWRfZGF0YSRiaW5hcnlfbGFiZWwsIAogICAgICBlc3RpbWF0ZSA9IGMoLngpLCAKICAgICAgZXZlbnRfbGV2ZWwgPSAic2Vjb25kIgogICAgKQogICkgfD4KICAgIGRwbHlyOjpiaW5kX3Jvd3MoLmlkID0gIm1ldGhvZCIpCiAgCiAgIyBzYXZlIGZvbGQgcHJlZGljdGlvbnMgZm9yIGZ1dHVyZSBpbnZlc3RpZ2F0aW9uCiAgdmFsaWRfcHJlZHNfbHNbW2ZvbGRdXSA8LSBjdl92YWxpZF9kYXRhX2FsbCB8PiAKICAgIGRwbHlyOjpiaW5kX2NvbHMocHJlZHNfbHMpCn0KCiMgZXhhbWluZSB2YWxpZGF0aW9uIGFjY3VyYWN5CnZhbGlkX3ByZWRzX2xzMSA8LSB2YWxpZF9wcmVkc19scwp2YWxpZF9hdXJvY19kZiA8LSBkcGx5cjo6YmluZF9yb3dzKHZhbGlkX2F1cm9jX2xzLCAuaWQgPSAiZm9sZCIpCm1lYW5fdmFsaWRfYXVyb2NfZGYgPC0gdmFsaWRfYXVyb2NfZGYgfD4gCiAgZHBseXI6OnN1bW1hcmlzZShkcGx5cjo6YWNyb3NzKC1mb2xkLCB+IG1lYW4oLngsIG5hLnJtID0gVFJVRSkpKQp2YWxpZF9hdXByY19kZiA8LSBkcGx5cjo6YmluZF9yb3dzKHZhbGlkX2F1cHJjX2xzLCAuaWQgPSAiZm9sZCIpCm1lYW5fdmFsaWRfYXVwcmNfZGYgPC0gdmFsaWRfYXVwcmNfZGYgfD4gCiAgZHBseXI6OnN1bW1hcmlzZShkcGx5cjo6YWNyb3NzKC1mb2xkLCB+IG1lYW4oLngsIG5hLnJtID0gVFJVRSkpKQoKIyBldmFsdWF0ZSBiZXN0IG1vZGVsIG9uIHRlc3Qgc2V0CnRyYWluX2RhdGEgPC0gdHJhaW5fZGF0YV9hbGwgfD4gCiAgZHBseXI6OnNlbGVjdCh0aWR5c2VsZWN0OjphbGxfb2Yoa2VlcF92YXJzKSkKdGVzdF9kYXRhIDwtIHRlc3RfZGF0YV9hbGwgfD4KICBkcGx5cjo6c2VsZWN0KHRpZHlzZWxlY3Q6OmFsbF9vZihrZWVwX3ZhcnMpKQpiZXN0X2ZpdCA8LSByYW5nZXI6OnJhbmdlcigKICBiaW5hcnlfbGFiZWwgfiAuLCBkYXRhID0gdHJhaW5fZGF0YSwgcHJvYmFiaWxpdHkgPSBUUlVFLCB2ZXJib3NlID0gRkFMU0UKKQp0ZXN0X3ByZWRzIDwtIHByZWRpY3QoYmVzdF9maXQsIHRlc3RfZGF0YSkkcHJlZGljdGlvbnNbLCAyXQp0ZXN0X2F1cm9jIDwtIHlhcmRzdGljazo6cm9jX2F1Y192ZWMoCiAgdHJ1dGggPSB0ZXN0X2RhdGEkYmluYXJ5X2xhYmVsLCAKICBlc3RpbWF0ZSA9IHRlc3RfcHJlZHMsIAogIGV2ZW50X2xldmVsID0gInNlY29uZCIKKQp0ZXN0X2F1cHJjIDwtIHlhcmRzdGljazo6cHJfYXVjX3ZlYygKICB0cnV0aCA9IHRlc3RfZGF0YSRiaW5hcnlfbGFiZWwsIAogIGVzdGltYXRlID0gdGVzdF9wcmVkcywgCiAgZXZlbnRfbGV2ZWwgPSAic2Vjb25kIgopCgphdXJvY19kZiA8LSBtZWFuX3ZhbGlkX2F1cm9jX2RmIHw+CiAgZHBseXI6OnJlbmFtZV93aXRoKH4gc3ByaW50ZigiVmFsaWRhdGlvbiAlcyBBVVJPQyIsIHN0cmluZ3I6OnN0cl90b190aXRsZSgueCkpKSB8PiAKICBkcGx5cjo6bXV0YXRlKAogICAgYFRlc3QgQVVST0NgID0gdGVzdF9hdXJvYwogICkKYXVwcmNfZGYgPC0gbWVhbl92YWxpZF9hdXByY19kZiB8PgogIGRwbHlyOjpyZW5hbWVfd2l0aCh+IHNwcmludGYoIlZhbGlkYXRpb24gJXMgQVVQUkMiLCBzdHJpbmdyOjpzdHJfdG9fdGl0bGUoLngpKSkgfD4gCiAgZHBseXI6Om11dGF0ZSgKICAgIGBUZXN0IEFVUFJDYCA9IHRlc3RfYXVwcmMKICApCgp2dGhlbWVzOjpwcmV0dHlfa2FibGUoCiAgdmFsaWRfYXVyb2NfZGYsIGNhcHRpb24gPSAiRm9sZC13aXNlIFZhbGlkYXRpb24gQVVST0MgZm9yIFZhcmlvdXMgTW9kZWxzIgopCnZ0aGVtZXM6OnByZXR0eV9rYWJsZSgKICBhdXJvY19kZiwgY2FwdGlvbiA9ICJPdmVyYWxsIEFVUk9DIFByZWRpY3Rpb24gUGVyZm9ybWFuY2UiCikKdnRoZW1lczo6cHJldHR5X2thYmxlKAogIHZhbGlkX2F1cHJjX2RmLCBjYXB0aW9uID0gIkZvbGQtd2lzZSBWYWxpZGF0aW9uIEFVUFJDIGZvciBWYXJpb3VzIE1vZGVscyIKKQp2dGhlbWVzOjpwcmV0dHlfa2FibGUoCiAgYXVwcmNfZGYsIGNhcHRpb24gPSAiT3ZlcmFsbCBBVVBSQyBQcmVkaWN0aW9uIFBlcmZvcm1hbmNlIgopCmBgYAoKPGRpdiBjbGFzcz0icGFuZWwgcGFuZWwtZGVmYXVsdCBwYWRkZWQtcGFuZWwiPgpIb3cgYXJlIHdlIGRvaW5nPyBBcmUgd2UgZG9pbmcgd2VsbD8gQW55IGNvbmNlcm5zPyAKPC9kaXY+CgojIyMgUG9zdC1ob2MgaW52ZXN0aWdhdGlvbnMgdjEgey50YWJzZXQgLnRhYnNldC1waWxscyAudGFic2V0LXBpbGxzLXNxdWFyZX0KCkxldCdzIGxvb2sgYXQgdGhlIGhlbGQtb3V0IGZvbGRzIHdoZXJlIHRoZSBtZXRob2RzIGRpZG4ndCBwZXJmb3JtIHNvIHdlbGwgYW5kIGNvbXBhcmUgdGhlIHByZWRpY3Rpb25zIGFjcm9zcyBtZXRob2RzLgoKYGBge3IgcmVzdWx0cyA9ICJhc2lzIn0KcGxvdF92YXJzIDwtIGMoCiAgImJpbmFyeV9sYWJlbCIsICJsb2dpc3RpYyIsICJsYXNzbyIsICJyaWRnZSIsICJyZiIsCiAgIkRGIiwgIkNGIiwgIkJGIiwgIkFGIiwgIkFOIiMsICJOREFJIiwgIlNEIiwgIkNPUlIiCikKZm9yIChmb2xkIGluIHVuaXF1ZSh0cmFpbl9kYXRhX2FsbCRibG9ja19pZCkpIHsKICBjYXQoc3ByaW50ZigiXG5cbiMjIyMgRm9sZCAlc1xuXG4iLCBmb2xkKSkKICBwcmVkc19kZiA8LSB2YWxpZF9wcmVkc19sc1tbZm9sZF1dCiAgcGx0X2xzIDwtIGxpc3QoKQogIGZvciAodmFyIGluIHBsb3RfdmFycykgewogICAgcGx0X2xzW1t2YXJdXSA8LSBwbG90X2Nsb3VkX2RhdGEocHJlZHNfZGYsIHZhcikKICB9CiAgcGx0IDwtIHBhdGNod29yazo6d3JhcF9wbG90cyhwbHRfbHMsIG5jb2wgPSA1KQogIHZ0aGVtZXM6OnN1YmNodW5raWZ5KAogICAgcGx0LCBpID0gc3ViY2h1bmtfaWR4LCBmaWdfd2lkdGggPSAxNCwgZmlnX2hlaWdodCA9IDYKICApCiAgc3ViY2h1bmtfaWR4IDwtIHN1YmNodW5rX2lkeCArIDEKfQoKIyBmb2xkIDwtICI4IiAjIGdvb2QgZm9sZAojIGZvbGQgPC0gIjEwIiAjICIxMSIgIyBiYWQgZm9sZAojIHByZWRzX2RmIDwtIHZhbGlkX3ByZWRzX2xzMVtbZm9sZF1dCiMgcGxvdF92YXJzIDwtIGMoCiMgICAjICJsYWJlbCIsCiMgICAiYmluYXJ5X2xhYmVsIiwgImxvZ2lzdGljIiwgImxhc3NvIiwgInJpZGdlIiwgInJmIiwKIyAgICJERiIsICJDRiIsICJCRiIsICJBRiIsICJBTiIjLCAiTkRBSSIsICJTRCIsICJDT1JSIgojICkKIyBwbHRfbHMgPC0gbGlzdCgpCiMgZm9yICh2YXIgaW4gcGxvdF92YXJzKSB7CiMgICBwbHRfbHNbW3Zhcl1dIDwtIHBsb3RfY2xvdWRfZGF0YShwcmVkc19kZiwgdmFyKQojIH0KIyBwbHQgPC0gcGF0Y2h3b3JrOjp3cmFwX3Bsb3RzKHBsdF9scywgbmNvbCA9IDUpCiMgcGx0CmBgYAoKIyMgTW9kZWxpbmcgdjIKCjxkaXYgY2xhc3M9InBhbmVsIHBhbmVsLWRlZmF1bHQgcGFkZGVkLXBhbmVsIj4KRW5naW5lZXJpbmcgZmVhdHVyZXMgYmFzZWQgdXBvbiBwcmlvciBvciBkb21haW4ga25vd2xlZGdlIGNhbiBncmVhdGx5IGltcHJvdmUgdGhlIGFjY3VyYWN5IG9mIG91ciBtb2RlbHMuIEluIFtTaGkgZXQgYWwuICgyMDA4KV0oaHR0cHM6Ly93d3cuanN0b3Iub3JnL3N0YWJsZS8yNzY0MDA4MSksIHRoZSBhdXRob3JzIGVuZ2luZWVyZWQgdGhyZWUgYWRkaXRpb25hbCBmZWF0dXJlczoKCi0gTkRBSTogbWVhc3VyZXMgZGlmZmVyZW5jZSBpbiByZWZsZWN0YW5jZSB2YWx1ZXMgYmV0d2VlbiBkaWZmZXJlbnQgcmFkaWFuY2UgYW5nbGVzIChvciBzcGVjdHJhbCBiYW5kcykKLSBDT1JSOiBtZWFzdXJlcyBjb3JyZWxhdGlvbiBiZXR3ZWVuIHRoZSBkaWZmZXJlbnQgcmFkaWFuY2UgYW5nbGVzCi0gU0Q6IG1lYXN1cmVzIHZhcmlhYmlsaXR5IGluIHJlZmxlY3RhbmNlIHZhbHVlcyBzdXJyb3VuZGluZyB0aGUgcGl4ZWwKCkxldCdzIHJlcGVhdCB0aGUgcHJldmlvdXMgYW5hbHlzaXMgaW5jbHVkaW5nIHRoZXNlIGVuZ2luZWVyZWQgZmVhdHVyZXMuCjwvZGl2PgoKYGBge3J9CmtlZXBfdmFycyA8LSBjKCJOREFJIiwgIlNEIiwgIkNPUlIiLCBrZWVwX3ZhcnMpCnRyYWluX2RhdGEgPC0gY2xvdWRfZGF0YSB8PgogIGRwbHlyOjpmaWx0ZXIoIShpbWFnZSAlaW4lIHRlc3RfaW1hZ2VfaWR4KSkKdGVzdF9kYXRhIDwtIGNsb3VkX2RhdGEgfD4KICBkcGx5cjo6ZmlsdGVyKGltYWdlICVpbiUgdGVzdF9pbWFnZV9pZHgpCgojIGV2YWx1YXRlIHZhbGlkYXRpb24gZXJyb3IgZm9yIHZhcmlvdXMgbW9kZWxzCnZhbGlkX2F1cm9jX2xzIDwtIGxpc3QoKQp2YWxpZF9hdXByY19scyA8LSBsaXN0KCkKdmFsaWRfcHJlZHNfbHMgPC0gbGlzdCgpCmZvciAoZm9sZCBpbiB1bmlxdWUodHJhaW5fZGF0YV9hbGwkYmxvY2tfaWQpKSB7CiAgIyBkbyBkYXRhIHNwbGl0CiAgY3ZfdHJhaW5fZGF0YV9hbGwgPC0gdHJhaW5fZGF0YV9hbGwgfD4gCiAgICBkcGx5cjo6ZmlsdGVyKGJsb2NrX2lkICE9ICEhZm9sZCkKICBjdl90cmFpbl9kYXRhIDwtIGN2X3RyYWluX2RhdGFfYWxsIHw+IAogICAgZHBseXI6OnNlbGVjdCh0aWR5c2VsZWN0OjphbGxfb2Yoa2VlcF92YXJzKSkKICBjdl92YWxpZF9kYXRhX2FsbCA8LSB0cmFpbl9kYXRhX2FsbCB8PiAKICAgIGRwbHlyOjpmaWx0ZXIoYmxvY2tfaWQgPT0gISFmb2xkKQogIGN2X3ZhbGlkX2RhdGEgPC0gY3ZfdmFsaWRfZGF0YV9hbGwgfD4gCiAgICBkcGx5cjo6c2VsZWN0KHRpZHlzZWxlY3Q6OmFsbF9vZihrZWVwX3ZhcnMpKQogIAogICMgZml0IGxvZ2lzdGljIHJlZ3Jlc3Npb24KICBsb2dfZml0IDwtIGdsbSgKICAgIGJpbmFyeV9sYWJlbCB+IC4sIGRhdGEgPSBjdl90cmFpbl9kYXRhLCBmYW1pbHkgPSAiYmlub21pYWwiCiAgKQogIGxvZ19wcmVkcyA8LSBwcmVkaWN0KGxvZ19maXQsIGN2X3ZhbGlkX2RhdGEsIHR5cGUgPSAicmVzcG9uc2UiKQogIAogICMgZml0IGFuZCBldmFsdWF0ZSBsYXNzbyByZWdyZXNzaW9uCiAgbGFzc29fZml0IDwtIGdsbW5ldDo6Y3YuZ2xtbmV0KAogICAgeCA9IGFzLm1hdHJpeChjdl90cmFpbl9kYXRhIHw+IGRwbHlyOjpzZWxlY3QoLWJpbmFyeV9sYWJlbCkpLAogICAgeSA9IGN2X3RyYWluX2RhdGEkYmluYXJ5X2xhYmVsLAogICAgZmFtaWx5ID0gImJpbm9taWFsIiwKICAgIGFscGhhID0gMSwKICAgIGZvbGRpZCA9IGFzLm51bWVyaWMoYXMuZmFjdG9yKGN2X3RyYWluX2RhdGFfYWxsJGJsb2NrX2lkKSkKICApCiAgbGFzc29fcHJlZHMgPC0gcHJlZGljdCgKICAgIGxhc3NvX2ZpdCwgCiAgICBhcy5tYXRyaXgoY3ZfdmFsaWRfZGF0YSB8PiBkcGx5cjo6c2VsZWN0KC1iaW5hcnlfbGFiZWwpKSwgCiAgICBzID0gImxhbWJkYS5taW4iLCAjIG9yICJsYW1iZGEuMXNlIgogICAgdHlwZSA9ICJyZXNwb25zZSIKICApCiAgCiAgIyBmaXQgYW5kIGV2YWx1YXRlIHJpZGdlIHJlZ3Jlc3Npb24KICByaWRnZV9maXQgPC0gZ2xtbmV0Ojpjdi5nbG1uZXQoCiAgICB4ID0gYXMubWF0cml4KGN2X3RyYWluX2RhdGEgfD4gZHBseXI6OnNlbGVjdCgtYmluYXJ5X2xhYmVsKSksCiAgICB5ID0gY3ZfdHJhaW5fZGF0YSRiaW5hcnlfbGFiZWwsCiAgICBmYW1pbHkgPSAiYmlub21pYWwiLAogICAgYWxwaGEgPSAwLAogICAgZm9sZGlkID0gYXMubnVtZXJpYyhhcy5mYWN0b3IoY3ZfdHJhaW5fZGF0YV9hbGwkYmxvY2tfaWQpKQogICkKICByaWRnZV9wcmVkcyA8LSBwcmVkaWN0KAogICAgcmlkZ2VfZml0LCAKICAgIGFzLm1hdHJpeChjdl92YWxpZF9kYXRhIHw+IGRwbHlyOjpzZWxlY3QoLWJpbmFyeV9sYWJlbCkpLCAKICAgIHMgPSAibGFtYmRhLm1pbiIsICMgb3IgImxhbWJkYS4xc2UiCiAgICB0eXBlID0gInJlc3BvbnNlIgogICkKICAKICAjIGZpdCByYW5kb20gZm9yZXN0CiAgcmZfZml0IDwtIHJhbmdlcjo6cmFuZ2VyKAogICAgYmluYXJ5X2xhYmVsIH4gLiwgZGF0YSA9IGN2X3RyYWluX2RhdGEsIAogICAgcHJvYmFiaWxpdHkgPSBUUlVFLCB2ZXJib3NlID0gRkFMU0UKICApCiAgcmZfcHJlZHMgPC0gcHJlZGljdChyZl9maXQsIGN2X3ZhbGlkX2RhdGEpJHByZWRpY3Rpb25zWywgMl0KICAKICAjIGV2YWx1YXRlIHByZWRpY3Rpb25zCiAgcHJlZHNfbHMgPC0gbGlzdCgKICAgICJsb2dpc3RpYyIgPSBsb2dfcHJlZHMsCiAgICAibGFzc28iID0gbGFzc29fcHJlZHMsCiAgICAicmlkZ2UiID0gcmlkZ2VfcHJlZHMsCiAgICAicmYiID0gcmZfcHJlZHMKICApCiAgdmFsaWRfYXVyb2NfbHNbW2ZvbGRdXSA8LSBwdXJycjo6bWFwKAogICAgcHJlZHNfbHMsCiAgICB+IHlhcmRzdGljazo6cm9jX2F1Y192ZWMoCiAgICAgIHRydXRoID0gY3ZfdmFsaWRfZGF0YSRiaW5hcnlfbGFiZWwsIAogICAgICBlc3RpbWF0ZSA9IGMoLngpLCAKICAgICAgZXZlbnRfbGV2ZWwgPSAic2Vjb25kIgogICAgKQogICkgfD4KICAgIGRwbHlyOjpiaW5kX3Jvd3MoLmlkID0gIm1ldGhvZCIpCiAgdmFsaWRfYXVwcmNfbHNbW2ZvbGRdXSA8LSBwdXJycjo6bWFwKAogICAgcHJlZHNfbHMsCiAgICB+IHlhcmRzdGljazo6cHJfYXVjX3ZlYygKICAgICAgdHJ1dGggPSBjdl92YWxpZF9kYXRhJGJpbmFyeV9sYWJlbCwgCiAgICAgIGVzdGltYXRlID0gYygueCksIAogICAgICBldmVudF9sZXZlbCA9ICJzZWNvbmQiCiAgICApCiAgKSB8PgogICAgZHBseXI6OmJpbmRfcm93cyguaWQgPSAibWV0aG9kIikKICAKICAjIHNhdmUgZm9sZCBwcmVkaWN0aW9ucyBmb3IgZnV0dXJlIGludmVzdGlnYXRpb24KICB2YWxpZF9wcmVkc19sc1tbZm9sZF1dIDwtIGN2X3ZhbGlkX2RhdGFfYWxsIHw+IAogICAgZHBseXI6OmJpbmRfY29scyhwcmVkc19scykKfQoKIyBleGFtaW5lIHZhbGlkYXRpb24gYWNjdXJhY3kKdmFsaWRfcHJlZHNfbHMyIDwtIHZhbGlkX3ByZWRzX2xzCm5ld192YWxpZF9hdXJvY19kZiA8LSBkcGx5cjo6YmluZF9yb3dzKHZhbGlkX2F1cm9jX2xzLCAuaWQgPSAiZm9sZCIpIHw+IAogIGRwbHlyOjpyZW5hbWVfd2l0aCh+IHNwcmludGYoIiVzIChuZXcpIiwgc3RyaW5ncjo6c3RyX3RvX3RpdGxlKC54KSkpIHw+IAogIGRwbHlyOjpyZW5hbWUoZm9sZCA9ICJGb2xkIChuZXcpIikgfD4gCiAgZHBseXI6OmxlZnRfam9pbih2YWxpZF9hdXJvY19kZiwgYnkgPSAiZm9sZCIpCm5ld19tZWFuX3ZhbGlkX2F1cm9jX2RmIDwtIG5ld192YWxpZF9hdXJvY19kZiB8PiAKICBkcGx5cjo6c3VtbWFyaXNlKGRwbHlyOjphY3Jvc3MoLWZvbGQsIH4gbWVhbigueCwgbmEucm0gPSBUUlVFKSkpCm5ld192YWxpZF9hdXByY19kZiA8LSBkcGx5cjo6YmluZF9yb3dzKHZhbGlkX2F1cHJjX2xzLCAuaWQgPSAiZm9sZCIpIHw+IAogIGRwbHlyOjpyZW5hbWVfd2l0aCh+IHNwcmludGYoIiVzIChuZXcpIiwgc3RyaW5ncjo6c3RyX3RvX3RpdGxlKC54KSkpIHw+IAogIGRwbHlyOjpyZW5hbWUoZm9sZCA9ICJGb2xkIChuZXcpIikgfD4gCiAgZHBseXI6OmxlZnRfam9pbih2YWxpZF9hdXByY19kZiwgYnkgPSAiZm9sZCIpCm5ld19tZWFuX3ZhbGlkX2F1cHJjX2RmIDwtIG5ld192YWxpZF9hdXByY19kZiB8PiAKICBkcGx5cjo6c3VtbWFyaXNlKGRwbHlyOjphY3Jvc3MoLWZvbGQsIH4gbWVhbigueCwgbmEucm0gPSBUUlVFKSkpCgojIGV2YWx1YXRlIGJlc3QgbW9kZWwgb24gdGVzdCBzZXQKdHJhaW5fZGF0YSA8LSB0cmFpbl9kYXRhX2FsbCB8PiAKICBkcGx5cjo6c2VsZWN0KHRpZHlzZWxlY3Q6OmFsbF9vZihrZWVwX3ZhcnMpKQp0ZXN0X2RhdGEgPC0gdGVzdF9kYXRhX2FsbCB8PgogIGRwbHlyOjpzZWxlY3QodGlkeXNlbGVjdDo6YWxsX29mKGtlZXBfdmFycykpCmJlc3RfZml0IDwtIHJhbmdlcjo6cmFuZ2VyKAogIGJpbmFyeV9sYWJlbCB+IC4sIGRhdGEgPSB0cmFpbl9kYXRhLCBwcm9iYWJpbGl0eSA9IFRSVUUsIHZlcmJvc2UgPSBGQUxTRQopCnRlc3RfcHJlZHMgPC0gcHJlZGljdChiZXN0X2ZpdCwgdGVzdF9kYXRhKSRwcmVkaWN0aW9uc1ssIDJdCnRlc3RfYXVyb2MgPC0geWFyZHN0aWNrOjpyb2NfYXVjX3ZlYygKICB0cnV0aCA9IHRlc3RfZGF0YSRiaW5hcnlfbGFiZWwsIAogIGVzdGltYXRlID0gdGVzdF9wcmVkcywgCiAgZXZlbnRfbGV2ZWwgPSAic2Vjb25kIgopCnRlc3RfYXVwcmMgPC0geWFyZHN0aWNrOjpwcl9hdWNfdmVjKAogIHRydXRoID0gdGVzdF9kYXRhJGJpbmFyeV9sYWJlbCwgCiAgZXN0aW1hdGUgPSB0ZXN0X3ByZWRzLCAKICBldmVudF9sZXZlbCA9ICJzZWNvbmQiCikKCm5ld19hdXJvY19kZiA8LSBuZXdfbWVhbl92YWxpZF9hdXJvY19kZiB8PgogIGRwbHlyOjpyZW5hbWVfd2l0aCh+IHNwcmludGYoIlZhbGlkYXRpb24gJXMgQVVST0MiLCBzdHJpbmdyOjpzdHJfdG9fdGl0bGUoLngpKSkgfD4gCiAgZHBseXI6Om11dGF0ZSgKICAgIGBUZXN0IEFVUk9DYCA9IHRlc3RfYXVyb2MKICApCm5ld19hdXByY19kZiA8LSBuZXdfbWVhbl92YWxpZF9hdXByY19kZiB8PgogIGRwbHlyOjpyZW5hbWVfd2l0aCh+IHNwcmludGYoIlZhbGlkYXRpb24gJXMgQVVQUkMiLCBzdHJpbmdyOjpzdHJfdG9fdGl0bGUoLngpKSkgfD4gCiAgZHBseXI6Om11dGF0ZSgKICAgIGBUZXN0IEFVUFJDYCA9IHRlc3RfYXVwcmMKICApCgp2dGhlbWVzOjpwcmV0dHlfa2FibGUoCiAgbmV3X3ZhbGlkX2F1cm9jX2RmLCBjYXB0aW9uID0gIkZvbGQtd2lzZSBWYWxpZGF0aW9uIEFVUk9DIGZvciBWYXJpb3VzIE1vZGVscyIKKQp2dGhlbWVzOjpwcmV0dHlfa2FibGUoCiAgbmV3X2F1cm9jX2RmLCBjYXB0aW9uID0gIk92ZXJhbGwgQVVST0MgUHJlZGljdGlvbiBQZXJmb3JtYW5jZSIKKQp2dGhlbWVzOjpwcmV0dHlfa2FibGUoCiAgbmV3X3ZhbGlkX2F1cHJjX2RmLCBjYXB0aW9uID0gIkZvbGQtd2lzZSBWYWxpZGF0aW9uIEFVUFJDIGZvciBWYXJpb3VzIE1vZGVscyIKKQp2dGhlbWVzOjpwcmV0dHlfa2FibGUoCiAgbmV3X2F1cHJjX2RmLCBjYXB0aW9uID0gIk92ZXJhbGwgQVVQUkMgUHJlZGljdGlvbiBQZXJmb3JtYW5jZSIKKQpgYGAKCiMjIyBQb3N0LWhvYyBpbnZlc3RpZ2F0aW9ucyB2MiB7LnRhYnNldCAudGFic2V0LXBpbGxzIC50YWJzZXQtcGlsbHMtc3F1YXJlfQoKTGV0J3MgbG9vayBhdCB0aGUgaGVsZC1vdXQgZm9sZHMgd2hlcmUgdGhlIG1ldGhvZHMgZGlkbid0IHBlcmZvcm0gc28gd2VsbCBhbmQgY29tcGFyZSB0aGUgcHJlZGljdGlvbnMgYWNyb3NzIG1ldGhvZHMuCgpgYGB7ciByZXN1bHRzID0gImFzaXMifQpwbG90X3ZhcnMgPC0gYygKICAiYmluYXJ5X2xhYmVsIiwgImxvZ2lzdGljIiwgImxhc3NvIiwgInJpZGdlIiwgInJmIiwKICAiREYiLCAiQ0YiLCAiQkYiLCAiQUYiLCAiQU4iLCAiTkRBSSIsICJTRCIsICJDT1JSIgopCmZvciAoZm9sZCBpbiB1bmlxdWUodHJhaW5fZGF0YV9hbGwkYmxvY2tfaWQpKSB7CiAgY2F0KHNwcmludGYoIlxuXG4jIyMjIEZvbGQgJXNcblxuIiwgZm9sZCkpCiAgcHJlZHNfZGYgPC0gdmFsaWRfcHJlZHNfbHNbW2ZvbGRdXQogIHBsdF9scyA8LSBsaXN0KCkKICBmb3IgKHZhciBpbiBwbG90X3ZhcnMpIHsKICAgIHBsdF9sc1tbdmFyXV0gPC0gcGxvdF9jbG91ZF9kYXRhKHByZWRzX2RmLCB2YXIpCiAgfQogIHBsdCA8LSBwYXRjaHdvcms6OndyYXBfcGxvdHMocGx0X2xzLCBuY29sID0gNSkKICB2dGhlbWVzOjpzdWJjaHVua2lmeSgKICAgIHBsdCwgaSA9IHN1YmNodW5rX2lkeCwgZmlnX3dpZHRoID0gMTQsIGZpZ19oZWlnaHQgPSA2CiAgKQogIHN1YmNodW5rX2lkeCA8LSBzdWJjaHVua19pZHggKyAxCn0KCiMgZm9sZCA8LSAiOCIgIyBnb29kIGZvbGQKIyBmb2xkIDwtICIxMCIgIyAiMTEiICMgYmFkIGZvbGQKIyBwcmVkc19kZiA8LSB2YWxpZF9wcmVkc19sczJbW2ZvbGRdXQojIHBsb3RfdmFycyA8LSBjKAojICAgIyAibGFiZWwiLAojICAgImJpbmFyeV9sYWJlbCIsICJsb2dpc3RpYyIsICJsYXNzbyIsICJyaWRnZSIsICJyZiIsCiMgICAiREYiLCAiQ0YiLCAiQkYiLCAiQUYiLCAiQU4iLCAiTkRBSSIsICJTRCIsICJDT1JSIgojICkKIyBwbHRfbHMgPC0gbGlzdCgpCiMgZm9yICh2YXIgaW4gcGxvdF92YXJzKSB7CiMgICBwbHRfbHNbW3Zhcl1dIDwtIHBsb3RfY2xvdWRfZGF0YShwcmVkc19kZiwgdmFyKQojIH0KIyBwbHQgPC0gcGF0Y2h3b3JrOjp3cmFwX3Bsb3RzKHBsdF9scywgbmNvbCA9IDUpCiMgcGx0CmBgYAoKPGRpdiBjbGFzcz0icGFuZWwgcGFuZWwtZGVmYXVsdCBwYWRkZWQtcGFuZWwiPgpIb3cgY2FuIHdlIGRvIHRoaXMgcG9zdC1ob2MgZXhwbG9yYXRpb24gaWYgdGhlcmUgYXJlIG1vcmUgdGhhbiA4IHZhcmlhYmxlcz8KPC9kaXY+CgojIEludGVycHJldGF0aW9ucwoKVG8gYmUgY29udGludWVkLi4uCg==